const Crawler = require("crawler")
const fs = require('fs')
// const http = require('http')
const Path = require('path')
const Axios = require('axios')

const request = require('request')

async function downloadImage (url, dir, index) {  
//   const url = 'https://unsplash.com/photos/AaEQmoufHLk/download?force=true'
  const path = Path.resolve(dir, `${index}.jpg`)
  request.get(url).pipe(fs.createWriteStream(path))
  return
  const writer = fs.createWriteStream(path)

  const response = await Axios({
    url,
    method: 'GET',
    responseType: 'stream'
  })

  response.data.pipe(writer)

  return new Promise((resolve, reject) => {
    writer.on('finish', resolve)
    writer.on('error', reject)
  })
}
/* 
const download = function(url, dir, index, cb) {
    if (!fs.existsSync(dir)){
        fs.mkdirSync(dir);
    }
    const path = Path.resolve(__dirname, 'downloads', dir, `${index}.jpg`)
    var file = fs.createWriteStream(path);
    var request = http.get(url, function(response) {
      response.pipe(file);
      file.on('finish', function() {
        file.close(cb);  // close() is async, call cb after close completes.
      });
    }).on('error', function(err) { // Handle errors
      fs.unlink(dest); // Delete the file async. (But we don't check the result)
      if (cb) cb(err.message);
    });
  }; */

var c = new Crawler({
    maxConnections : 10,
    // This will be called for each crawled page
    callback : function (error, res, done) {
        if(error){
            console.log(error);
        }else{
            var $ = res.$;
            // $ is Cheerio by default
            //a lean implementation of core jQuery designed specifically for the server
            console.log($("title").text());
        }
        done();
    }
})

const listCallback = function (error, res, done) {
        if(error){
            console.log(error);
        }else{
            var $ = res.$;
            // $ is Cheerio by default
            //a lean implementation of core jQuery designed specifically for the server
            console.log($(".archives .url").find('a').length)
            $(".archives .url").find('a').toArray().forEach((element, index) => {
                if (index > 0) { return }
                // https://stackoverflow.com/questions/3486359/regex-to-extract-substring-returning-2-results-for-some-reason
                const result = $(element).attr('href').match(/(?<=\/)\d+(?=$)/ig)
                const id = result[0]
                console.log(id)
                c.queue([{
                    uri: `http://adr.meizitu.net/wp-json/wp/v2/i?id=${id}`,
                    jQuery: false,
                    callback: detailCallback,
                    // https://github.com/bda-research/node-crawler#custom-parameters
                    title: $(element).text()
                }])
            })
        }
        done();
    }

const detailCallback = function (error, res, done) {
        if(error){
            console.log(error);
        }else{
            const data = JSON.parse(res.body)
            const content = data.content

            const title = res.options.title
            console.log(title)
            const dir = Path.resolve(__dirname, 'downloads', title)
            console.log(dir)
            if (!fs.existsSync(dir)){
                fs.mkdirSync(dir)
            }

            content.split(',').forEach((image, index) => {
                console.log(image)
                // download(image, res.options.title, index, () => {})
                downloadImage(image, dir, index)
            })
            // var $ = res.$;
            // $ is Cheerio by default
            //a lean implementation of core jQuery designed specifically for the server
            /* console.log($(".archives .url").find('a').length)
            $(".archives .url").find('a').toArray().forEach(element => {
                // console.log(element.href)
                const result = $(element).attr('href').match(/(?<=\/)\d+(?=$)/ig)
                console.log(result[0])
            }) */
        }
        done();
    }
c.queue([{uri: 'https://www.mzitu.com/all/',
    callback : listCallback
}])

// https://www.mzitu.com/all/ 专辑列表页面 近期图片
// https://www.mzitu.com/old/ 专辑列表页面 早期图片
// 在列表页面的浏览器的元素审查中搜索 .archives .url a
// 3003 + 2264

// wordpress wp-json api
// http://adr.meizitu.net/wp-json/wp/v2/posts?page=53&per_page=100 专辑列表接口
// http://adr.meizitu.net/wp-json/wp/v2/i?id=41937 专辑详情接口

// 接口返回的图片个数和这个专辑详情页面的分页数是一样的，并且图片内容也是一样的
// https://www.mzitu.com/41937 专辑详情页面